//抓取麦田官网
package main

import (
	//"bytes"
	"fmt"
	goq "lib/goquery"
	"lib/request"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	main_domain = "" //主域名
	LOG         = ""
	NumCPU      = runtime.NumCPU()
	waitgroup   = &sync.WaitGroup{}
)

func init() {
	runtime.GOMAXPROCS(NumCPU)

	main_domain = "http://www.maitian.cn"
	LOG = "maitian"
}

func main() {
	getAgentPage()
	s(time.Now().Format("2006-01-02 15:04:05"), "done.")
}

//获取经纪人分页
func getAgentPage() {
	url := fmt.Sprintf("%s/bkesf/", main_domain)
	doc, e := getDoc(url)
	if e != nil {
		return
	}
	tmp_recordCount := filterStr(doc.Find("p:contains('共找到房产顾问') span").Text())
	if tmp_recordCount == "" {
		return
	}

	recordCount, e := strconv.Atoi(tmp_recordCount)
	if e != nil {
		return
	}

	pageSize := 10
	pageCount := math.Ceil(float64(recordCount) / float64(pageSize))

	var curpage float64
	for curpage = 1; curpage <= pageCount; curpage++ {
		tmpurl := fmt.Sprintf("%sPG%0.0f", url, curpage)
		getAgentList(tmpurl)
	}
}

//获取经纪人列表
func getAgentList(url string) {
	doc, e := getDoc(url)
	if e != nil {
		errlog(url + e.Error())
		return
	}

	var arr = []string{}

	doc.Find(".list_wrap .clearfix").Each(func(i int, sec *goq.Selection) {
		href, ok := sec.Find(".agent_man dt > a").Attr("href")
		if ok {
			href = fmt.Sprintf("%s%s", main_domain, href)
			arr = append(arr, href)
		}
	})
	for i := 0; i < len(arr); i += NumCPU {
		for j := i; j < len(arr) && j < i+NumCPU; j++ {
			waitgroup.Add(1)
			go getAgentDetail(arr[j])
		}
		waitgroup.Wait()
		time.Sleep(time.Second * 3)
	}
}

//获取经纪人详情
func getAgentDetail(url string) {
	defer waitgroup.Done()
	doc, e := getDoc(url)
	if e != nil {
		errlog(url + e.Error())
		return
	}

	var (
		companyname   string //公司
		shopname      string //店铺
		agentname     string //经纪人姓名
		mobile        string //手机
		district      string //服务区县
		block         string //服务热点
		communtiyname string //服务楼盘
	)
	companyname = "麦田房产"

	font12 := filterStr(doc.Find(".agent_left ul .font12").Text())
	font12 = strings.Replace(font12, string([]byte{229, 167, 147, 194, 160, 194, 160, 229, 144, 141, 239, 188, 154}), "", -1)
	font12 = strings.Replace(font12, string([]byte{32}), "", -1)
	font12 = strings.Replace(font12, string([]byte{194, 160}), "", -1)
	tmp_arr := strings.Split(font12, "(")

	agentname = tmp_arr[0]
	if agentname == "" {
		agentname = "--"
	}

	tmp_arr = strings.Split(tmp_arr[1], "所属门店：")
	tmp_arr = strings.Split(tmp_arr[1], "联系方式：")

	if len(tmp_arr) == 2 {
		shopname, mobile = tmp_arr[0], tmp_arr[1]
	} else {
		shopname, mobile = "--", "--"
	}

	communtiyname_arr := doc.Find(".agent_left ul .distance a").Not(".spac").Map(func(i int, sec *goq.Selection) string {
		return filterStr(sec.Text())
	})
	communtiyname = strings.Join(communtiyname_arr, ",")
	if communtiyname == "" {
		communtiyname = "--"
	}

	tmpstr := filterStr(doc.Find(".agent_left ul .distance").Not("a").Text())
	tmp_arr = strings.Split(tmpstr, "熟悉小区")
	tmpstr = tmp_arr[0]
	tmpstr = strings.Replace(tmpstr, "服务商圈：[", "", -1)
	tmpstr = strings.Replace(tmpstr, "]", "", -1)
	tmp_arr = strings.Split(tmpstr, string([]byte{32}))
	if len(tmp_arr) == 2 {
		district, block = tmp_arr[0], tmp_arr[1]
	} else {
		district, block = "--", "--"
	}

	str := companyname
	str += "#" + shopname
	str += "#" + agentname
	str += "#" + mobile
	str += "#" + district
	str += "#" + block
	str += "#" + communtiyname

	outlog(str)
}

//读取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := request.Get(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "#", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//debuglog
func debuglog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_debug.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
